1. Calibrate the Camera¶
Compute the camera calibration matrix and distortion coefficients given a set of chessboard images.
In [1]:
# import modules for this project
import cv2
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
%matplotlib inline
In [2]:
import glob
# Read in and make a list of calibration chessboard images
chessboard_images = glob.glob('./camera_cal/calibration*.jpg') # Chessboard images
test_images = glob.glob('./test_images/*.jpg') # Test images
In [3]:
# Arrays to store object points and image points from all the images
objpoints = [] # 3D points in real world space
imgpoints = [] # 2D points in image plane
# Prepare object points, like (0, 0, 0), (1, 0, 0) ..., (8, 5, 0)
objp = np.zeros((6*9, 3), np.float32)
# Generate the x and y coordinates and shape them into two columns
objp[:,:2] = np.mgrid[0:9,0:6].T.reshape(-1,2)
for fname in chessboard_images:
# read in each image
img = mpimg.imread(fname)
# Convert image to grayscale
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
# Find the chessboard corners
ret, corners = cv2.findChessboardCorners(gray, (9,6), None)
# If coners are found, add object points, image points
if ret == True:
imgpoints.append(corners)
objpoints.append(objp)
# draw detected corners on the chessboard image and store it
name = './output_images/draw_corners/' + fname.split('/')[-1]
img_draw = cv2.drawChessboardCorners(img, (9,6), corners, ret)
plt.imshow(img_draw)
plt.savefig(name)
cv2.destroyAllWindows()
In [4]:
# Save the objpoints and imgpoints arrays
np.save('./output_images/imgpoints.npy', imgpoints)
np.save('./output_images/objpoints.npy', objpoints)
2. Correct for Image Distortion¶
Apply a distortion correction to raw images.
In [5]:
def cal_undistort(img):
"""
The function performs the camera calibration
and images distortion correction,
returns the undistorted image.
[img] a distorted 2D image
[objpoints] the coordinates of the corners in undistorted 3D images
[imgpoints] the coordinates of the corners in distorted 2D images.
"""
# Load objpoints and imgpoints
imgpoints = np.load('./output_images/imgpoints.npy')
objpoints = np.load('./output_images/objpoints.npy')
# Convert to grayscale
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
# Find the camera matrix and distortion coefficients to transform 3D image points to 2D image points
ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[::-1], None, None)
# Undistort the input image
undist = cv2.undistort(img, mtx, dist, None, mtx)
return undist
In [6]:
img_chessboard = mpimg.imread('./camera_cal/calibration3.jpg')
img_lane = mpimg.imread(test_images[0])
In [7]:
undistorted1 = cal_undistort(img_chessboard)
undistorted2 = cal_undistort(img_lane)
In [8]:
f, arr = plt.subplots(2, 2, figsize=(8, 6))
f.tight_layout()
arr[0,0].set_title('Original Image', fontsize=20)
arr[0,0].imshow(img_chessboard)
arr[1,0].set_title('Original Image', fontsize=20)
arr[1,0].imshow(img_lane)
arr[0,1].set_title('undistorted Image', fontsize=20)
arr[0,1].imshow(undistorted1)
arr[1,1].set_title('undistorted Image', fontsize=20)
arr[1,1].imshow(undistorted2)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)
plt.savefig('./output_images/undistorted_test.jpg')
3. Implement a Color & Gradient Threshold¶
Use color transform, gradients, etc., to create a threshold binary image.
Note: Make sure you use the correct grayscale conversion depending on how you've read in your images. Use cv2.COLOR_RGB2GRAY if you've read in an image using mpimg.imread(). Use cv2.COLOR_BGR2GRAY if you've read in an image using cv2.imread().
3.1 Load and display original test images¶
Those test images are used to:
- check my threshold functions
- tune and get the best parameter values for my threshold functions
3.1.1 Import snapshots of project_video.mp4 as test images¶
Import four snapshots of the project_video.mp4 which failed in my 1st submission as additional test images.
In [9]:
vidcap = cv2.VideoCapture('./project_video.mp4')
times = [23000, 28000, 23750, 42000]
i = 0
for t in times:
vidcap.set(cv2.CAP_PROP_POS_MSEC, t) # just cue to t/1000 sec. position
success, image = vidcap.read()
if success:
i+=1
name = './test_images/snapshot' + str(i) + '.jpg'
cv2.imwrite(name,image)
3.1.2 Load all test images¶
In [10]:
# load all test images
images= []
names = []
for fname in test_images:
# load and append test image
img = mpimg.imread(fname)
images.append(img)
# pick image name out of file path
name = fname.split('/')[-1].split('.')[0]
names.append(name)
In [11]:
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
r = i//4 # row of image
c = i - r*4 # colomn of image
arr[r,c].imshow(images[i], cmap='gray')
arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/thresholds/load_images.jpg')
3.2 Color Thresholds¶
3.2.1 HLS threshold¶
As to color space, HLS can be more robust than RGB. Here I'll read in the same original image (straight lines1), convert to sparated H,L and S channels to get the following results:
In [12]:
hls_0 = cv2.cvtColor(img_lane, cv2.COLOR_RGB2HLS)
H_0 = hls_0[:,:,0]
L_0 = hls_0[:,:,1]
S_0 = hls_0[:,:,2]
f, arr = plt.subplots(1, 3, figsize=(24, 9))
f.tight_layout()
arr[0].set_title('H_channel', fontsize=30)
arr[0].imshow(H_0, cmap='gray')
arr[1].set_title('L_channel', fontsize=30)
arr[1].imshow(L_0, cmap='gray')
arr[2].set_title('S_channel', fontsize=30)
arr[2].imshow(S_0, cmap='gray')
plt.savefig('./output_images/thresholds/HLS_compare.jpg')
The S channel picks up the lines well, so I'll apply a threshold on it.
In [13]:
def hls_select(img, thresh=(0, 255)):
"""
The fucntion applies threshold on the S-channel of HLS.
"""
# Select the S channel as it picks up the lines well
hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
s_channel = hls[:,:,2]
# Apply a threshold on S channel
binary_output = np.zeros_like(s_channel)
binary_output[(s_channel > thresh[0]) & (s_channel <= thresh[1])] = 1
return binary_output
I tune the parameter as hls_select(img, thresh=(80, 255))
In [14]:
# Test function and tune parameters to get ideal output
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
binary_output = hls_select(images[i], thresh=(80, 255))
r = i//4 # row of image
c = i - r*4 # colomn of image
arr[r,c].imshow(binary_output, cmap='gray')
arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/thresholds/HLS.jpg')
3.2.2 RGB threshold¶
The HLS threshold works pretty good, but there're still some shadow areas in white after using hls_select(), such as frame42 and test4. Those white shadow area will be noise in the following lane line detection.
I'll take use rgb threshold to convert those shadow into black. Thus the RGB threshold should keep lane line in white and shadows which not detected by HLS threshold in black, no matter whether other areas are in white or black.
In [15]:
rgb_0 = img_lane
R_0 = rgb_0[:,:,0]
G_0 = rgb_0[:,:,1]
B_0 = rgb_0[:,:,2]
f, arr = plt.subplots(1, 3, figsize=(24, 9))
f.tight_layout()
arr[0].set_title('R_channel', fontsize=30)
arr[0].imshow(R_0, cmap='gray')
arr[1].set_title('G_channel', fontsize=30)
arr[1].imshow(G_0, cmap='gray')
arr[2].set_title('B_channel', fontsize=30)
arr[2].imshow(B_0, cmap='gray')
plt.savefig('./output_images/thresholds/HLS_compare.jpg')
The R channel picks up the lines well, so I'll apply a threshold on it.
In [16]:
def rgb_select(img, thresh=(0, 255)):
"""
The fucntion applies threshold on the R-channel of RGB.
"""
# Select the R channel as it picks up the lines well
r_channel = img[:,:,0]
# Apply a threshold on S channel
binary_output = np.zeros_like(r_channel)
binary_output[(r_channel > thresh[0]) & (r_channel <= thresh[1])] = 1
return binary_output
I tune the parameter as rgb_select(img, thresh=(50, 255))
In [17]:
# Test function and tune parameters to get ideal output
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
binary_output = rgb_select(images[i], thresh=(50, 255))
r = i//4 # row of image
c = i - r*4 # colomn of image
arr[r,c].imshow(binary_output, cmap='gray')
arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/thresholds/RGB.jpg')
3.2.3 Combine HLS and RGB threshold¶
To summarize, I tune all color threshold parameters as below:
hls_select(img, thresh=(80, 255))rgb_select(img, thresh=(50, 255))
In [18]:
def combine_color(img):
"""
The function combines S_channel and R_channel to set color thresholds.
"""
# Apply each of the thresholding functions
s_binary = hls_select(img, thresh=(80, 255))
r_binary = rgb_select(img, thresh=(50, 255))
combined = np.zeros_like(s_binary)
combined[(s_binary==1) & (r_binary==1)] = 1
return combined
In [19]:
# Test function and tune parameters to get ideal output
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
binary_output = combine_color(images[i])
r = i//4 # row of image
c = i - r*4 # colomn of image
arr[r,c].imshow(binary_output, cmap='gray')
arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/thresholds/combine_color.jpg')
In [20]:
def abs_sobel_thresh(img, orient='x', thresh=(0,255)):
"""
The function applies Sobel x and y,
then takes an absolute value and applies a threshold.
"""
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
# Calculate directional gradient
if orient == 'x':
abs_sobel = np.absolute(cv2.Sobel(gray, cv2.CV_64F, 1, 0))
elif orient == 'y':
abs_sobel = np.absolute(cv2.Sobel(gray, cv2.CV_64F, 0, 1))
# Rescale back to 8 bit integer
scaled_sobel = np.uint8(255*abs_sobel/np.max(abs_sobel))
# Apply threshold
binary_output = np.zeros_like(scaled_sobel)
binary_output[(scaled_sobel >= thresh[0]) & (scaled_sobel <= thresh[1])] = 1
return binary_output
To apply threshold on the absolute value of the gradient in x direction, I tune the parameter as abs_sobel_thresh(img, orient='x', thresh=(20, 200))
In [21]:
# Test function and tune parameters to get ideal output
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
binary_output_i = abs_sobel_thresh(images[i], orient='x', thresh=(20, 200))
r = i//4 # row of image
c = i - r*4 # colomn of image
arr[r,c].imshow(binary_output_i, cmap='gray')
arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/thresholds/abs_sobelx_thresh.jpg')
To apply threshold on the absolute value of the gradient in y direction, I tune the parameter as abs_sobel_thresh(img, orient='y', thresh=(20, 200))
In [22]:
# Test function and tune parameters to get ideal output
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
binary_output_i = abs_sobel_thresh(images[i], orient='y', thresh=(20, 200))
r = i//4 # row of image
c = i - r*4 # colomn of image
arr[r,c].imshow(binary_output_i, cmap='gray')
arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/thresholds/abs_sobely_thresh.jpg')
3.3.2 Magnitude of the Gradient¶
In [23]:
def mag_thresh(img, sobel_kernel, thresh=(0, 255)):
"""
The function returns the magnitude of the gradient
for a given sobel kernel size and threshold values.
"""
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
# Calculate gardient magnitude
sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
gradmag = np.sqrt(sobelx**2, sobely**2)
# Rescale to 8 bit
scale_factor = np.max(gradmag)/255
gradmag = (gradmag/scale_factor).astype(np.uint8)
# Apply threshold
binary_output = np.zeros_like(gradmag)
binary_output[(gradmag >= thresh[0]) & (gradmag <= thresh[1])] = 1
return binary_output
I tune the parameter as mag_thresh(img, sobel_kernel=3, thresh=(30, 100))
In [24]:
# Test function and tune parameters to get ideal output
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
binary_output = mag_thresh(images[i], sobel_kernel=3, thresh=(30, 100))
r = i//4 # row of image
c = i - r*4 # colomn of image
arr[r,c].imshow(binary_output, cmap='gray')
arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/thresholds/mag_thresh.jpg')
3.3.3 Direction of the Gradient¶
In [25]:
def dir_threshold(img, sobel_kernel, thresh=(0, np.pi/2)):
"""
The function applies Sobel x and y, then compute the
direction of the gradient and applies a threshold.
"""
#gray = hls_select(img, thresh=(90, 255))
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
# Calculate gradient direction
sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
absgraddir = np.arctan2(np.absolute(sobely), np.absolute(sobelx))
# Apply threshold
binary_output = np.zeros_like(absgraddir)
binary_output[(absgraddir >= thresh[0]) & (absgraddir <= thresh[1])] = 1
return binary_output
I tune the parameter as dir_threshold(img, sobel_kernel=9, thresh=(0.7, 1.2))
In [26]:
# Test function and tune parameters to get ideal output
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
binary_output = dir_threshold(images[i], sobel_kernel=9, thresh=(0.7, 1.2))
r = i//4 # row of image
c = i - r*4 # colomn of image
arr[r,c].imshow(binary_output, cmap='gray')
arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/thresholds/dir_threshold.jpg')
3.3.4 Combing Gradient Thresholds¶
To summarize, I tune all gradient threshold parameters as below:
abs_sobel_thresh(img, orient='x', thresh=(20, 200))abs_sobel_thresh(img, orient='x', thresh=(20, 200))mag_thresh(img, sobel_kernel=3, thresh=(30, 100))dir_threshold(img, sobel_kernel=9, thresh=(0.7, 1.2))
In [27]:
def combine_gradient(img):
"""
The function combines gradient, magnitude of gradient and
direction of gradient thresholds.
"""
# Apply each of the thresholding functions
gradx = abs_sobel_thresh(img, orient='x', thresh=(20, 200))
grady = abs_sobel_thresh(img, orient='y', thresh=(20, 200))
mag_binary = mag_thresh(img, sobel_kernel=3, thresh=(30, 100))
dir_binary = dir_threshold(img, sobel_kernel=9, thresh=(0.7, 1.2))
combined = np.zeros_like(dir_binary)
combined[((gradx==1) & (grady==1)) | ((mag_binary==1) & (dir_binary==1))] = 1
return combined
In [28]:
# Test function
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
binary_output = combine_gradient(images[i])
r = i//4 # row of image
c = i - r*4 # colomn of image
arr[r,c].imshow(binary_output, cmap='gray')
arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/thresholds/combine_gradient.jpg')
3.4 Combining Color and Gradient Thresholds¶
In [29]:
def color_grad(img):
"""
The function combines HLS color threshold and multiple gradient thresholds.
"""
color_binary = combine_color(img)
gradient_binary = combine_gradient(img)
combined_binary = np.zeros_like(color_binary)
combined_binary[(color_binary == 1) | (gradient_binary == 1)] = 1
return combined_binary
In [30]:
# Test function
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
binary_output = color_grad(images[i])
r = i//4 # row of image
c = i - r*4 # colomn of image
arr[r,c].imshow(binary_output, cmap='gray')
arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/thresholds/color_grad.jpg')
In [31]:
# Run the function
combine_binary = color_grad(img_lane)
# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(img_lane)
ax1.set_title('Original Image', fontsize=50)
ax2.imshow(combine_binary, cmap='gray')
ax2.set_title('Color & Gradient', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)
plt.savefig('./output_images/thresholds/thresholds.jpg')
4. Warp the Image Using Perspective Transform¶
Apply a perspective transform to rectify binary image("birds-eye view").
4.1 Choose Source and Destination Points¶
In [32]:
img_size = (1280, 720)
src = np.int32(
[[(img_size[0] * 5 / 6) + 60, img_size[1]],
[(img_size[0] / 2 + 65), img_size[1] / 2 + 100],
[(img_size[0] / 2) - 60, img_size[1] / 2 + 100],
[((img_size[0] / 6) - 10), img_size[1]]])
dst = np.int32(
[[(img_size[0] * 5 / 6), img_size[1]],
[(img_size[0] * 5 / 6), 0],
[(img_size[0] / 6), 0],
[(img_size[0] / 6), img_size[1]]])
In [33]:
src
Out[33]:
In [34]:
dst
Out[34]:
Apply the source (original) coordinates src and destination (desired or warped) coordinates dst above in the following warper function.
In [35]:
def warper(img):
"""
Define caliation box in source (original)
and destination (desired or warped) coordinates.
"""
img_size = (img.shape[1], img.shape[0])
# Four source coordinates
src = np.float32([[1126, 720],
[ 705, 460],
[ 580, 460],
[ 203, 720]])
# Four desired coordinates
dst = np.float32([[1066, 720],
[1066, 0],
[ 213, 0],
[ 213, 720]])
# Compute the perspective transform M
M = cv2.getPerspectiveTransform(src, dst)
# Compute the inverse perspective transform Minv
Minv = cv2.getPerspectiveTransform(dst, src)
# Create warped image - uses linear interpolation
warped = cv2.warpPerspective(img, M, img_size, flags=cv2.INTER_LINEAR)
return warped, M, Minv
4.2 Verify Source and Destination Points¶
I verified that my perspective transform was working as expected by drawing the src and dst points onto two test images with straight lines and their warped counterparts respectively to verify that the lines appear parallel in their corresponding warped image.
In [36]:
# Read test images with straight lines
straight_lines1 = mpimg.imread('./test_images/straight_lines1.jpg')
straight_lines2 = mpimg.imread('./test_images/straight_lines2.jpg')
# Undistort the images
undist_sl1 = cal_undistort(straight_lines1)
undist_sl2 = cal_undistort(straight_lines2)
# Warp the undistorted images
warp_sl1, _, _ = warper(undist_sl1)
warp_sl2, _, _ = warper(undist_sl2)
In [37]:
# Run the function
combine_binary = color_grad(img_lane)
# Plot the result
f, arr = plt.subplots(2, 2, figsize=(10, 8))
f.tight_layout()
arr[0,0].imshow(undist_sl1, extent=(0,1280,720,0))
arr[0,0].plot([1126, 705, 580, 203],
[ 720, 460, 460, 720],'r-',linewidth=2)
arr[0,0].set_title('Udistorted Image_1 with [src] drawn', fontsize=18)
arr[0,1].imshow(undist_sl2, extent=(0,1280,720,0))
arr[0,1].plot([1126, 705, 580, 203],
[ 720, 460, 460, 720],'r-',linewidth=2)
arr[0,1].set_title('Udistorted Image_2 with [src] drawn', fontsize=18)
arr[1,0].imshow(warp_sl1, cmap='gray', extent=(0,1280,720,0))
arr[1,0].plot([213, 213], [ 0, 720],'r-',linewidth=2)
arr[1,0].plot([1066,1066], [ 0, 720],'r-',linewidth=2)
arr[1,0].set_title('Warped result_1 with [dst] drawn', fontsize=18)
arr[1,1].imshow(warp_sl2, cmap='gray', extent=(0,1280,720,0))
arr[1,1].plot([213, 213], [ 0, 720],'r-',linewidth=2)
arr[1,1].plot([1066,1066], [ 0, 720],'r-',linewidth=2)
arr[1,1].set_title('Warped result_2 with [dst] drawn', fontsize=18)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)
plt.savefig('./output_images/src&dst_drawn.jpg')
4.3 Warp a Binary Image¶
I pick up all undistortion, threshold and warper function together to get a function warp_camera(img_camera) which take a original distorted image as input and output undistorted and warped binary image in bird-eye perspection.
In [38]:
def warp_camera(img_camera):
"""
The input is the original image taken by camera.
The function undistort the original image, apply a color&gradient threshold
on the undistorted image and warp the binary image in bird-eye perspection.
"""
img_undistort = cal_undistort(img_camera)
img_threshold = color_grad(img_undistort)
img_warped, M, Minv = warper(img_threshold)
return img_undistort, img_warped, M, Minv
In [39]:
# Test function
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
_, warped_i, _, _ = warp_camera(images[i])
r = i//4 # row of image
c = i - r*4 # colomn of image
arr[r,c].imshow(warped_i, cmap='gray')
arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/warped_binary_image.jpg')
In [40]:
def window_mask(width, height, img_ref, center,level):
"""
The function is used to draw window areas.
"""
output = np.zeros_like(img_ref)
output[int(img_ref.shape[0]-(level+1)*height):int(img_ref.shape[0]-level*height),max(0,int(center-width/2)):min(int(center+width/2),img_ref.shape[1])] = 1
return output
5.1.1 Unidirectional Window Fitting¶
In [41]:
def find_window_centroids(binary_warped, window_width, window_height, margin):
"""
The function finds all the left and right window centroids
for each level in the given binary image.
"""
window_centroids = [] # Store the (left, right) window centroid positions per level
window = np.ones(window_width) # Create out window template that we will use for convolutions
# Sum quarter bottom of image to get slice, could use a different ratio
l_sum = np.sum(binary_warped[int(3*binary_warped.shape[0]/4):,:int(binary_warped.shape[1]/2)], axis=0)
l_center = np.argmax(np.convolve(window,l_sum))-window_width/2
r_sum = np.sum(binary_warped[int(3*binary_warped.shape[0]/4):,int(binary_warped.shape[1]/2):], axis=0)
r_center = np.argmax(np.convolve(window,r_sum))-window_width/2+int(binary_warped.shape[1]/2)
# Add what we found for the first layer
window_centroids.append((l_center,r_center))
# Go through each layer looking for max pixel locations
for level in range(1,int(binary_warped.shape[0]/window_height)):
# Convolve the window into the vertical slice of the image
layer_bottom = int(binary_warped.shape[0]-(level+1)*window_height)
layer_top = int(binary_warped.shape[0]-level*window_height)
image_layer = np.sum(binary_warped[layer_bottom:layer_top, :], axis=0)
conv_signal = np.convolve(window, image_layer)
# Use window_width/2 as offset because convolution signal reference is at right side of window, not center of window
offset = window_width/2
# Find the best left centroid by using past left center as a reference
l_min_index = int(max(l_center+offset-margin,0))
l_max_index = int(min(l_center+offset+margin,binary_warped.shape[1]))
l_center = np.argmax(conv_signal[l_min_index:l_max_index])+l_min_index-offset
# Find the best right centroid by using past right center as a reference
r_min_index = int(max(r_center+offset-margin,0))
r_max_index = int(min(r_center+offset+margin,binary_warped.shape[1]))
r_center = np.argmax(conv_signal[r_min_index:r_max_index])+r_min_index-offset
# Add what we found for that layer
window_centroids.append((l_center,r_center))
return window_centroids
In [42]:
def mark_centroids(binary_warped, window_centroids, window_width, window_height):
"""
The function find and mark left and right centroids.
"""
# If we found any window centers
if len(window_centroids) > 0:
# Points used to draw all the left and right windows
l_points = np.zeros_like(binary_warped)
r_points = np.zeros_like(binary_warped)
# Go through each level and draw the windows
for level in range(0,len(window_centroids)):
# Window_mask is a function to draw window areas
l_mask = window_mask(window_width,window_height,binary_warped,window_centroids[level][0],level)
r_mask = window_mask(window_width,window_height,binary_warped,window_centroids[level][1],level)
# Add graphic points from window mask here to total pixels found
l_points[(l_points == 255) | ((l_mask == 1))] = 255
r_points[(r_points == 255) | ((r_mask == 1))] = 255
# Draw the results
template = np.array(r_points+l_points,np.uint8) # add both left and right window pixels together
zero_channel = np.zeros_like(template) # create a zero color channel
template = np.array(cv2.merge((zero_channel,template,zero_channel)),np.uint8) # make window pixels green
warpage = np.dstack((binary_warped, binary_warped, binary_warped))*255 # making the original road pixels 3 color channels
output = cv2.addWeighted(warpage, 1, template, 0.5, 0.0) # overlay the original road image with window results
# If no window centers found, just display original road image
else:
output = np.array(cv2.merge((binary_warped, binary_warped, binary_warped)),np.uint8)
return output
In [43]:
# Window settings
window_width = 50
window_height = 80 # Break image into 9 vertical layers since image height is 720
margin = 50 # How much to slide left and right for searching
In [44]:
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
_, warped_i, _, _ = warp_camera(images[i])
window_centroids_i = find_window_centroids(warped_i, window_width, window_height, margin)
mark_centroids_i = mark_centroids(warped_i, window_centroids_i, window_width, window_height)
r = i//4 # row of image
c = i - r*4 # colomn of image
arr[r,c].imshow(mark_centroids_i, cmap='gray')
arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/window_fitting_result.jpg')
5.1.2 Bidirectional Window Fitting¶
As the output images shown above, the find_window_centroids works well in the front half part of the levels but tends to be out of order in the back half part of the levels.
Thus I decide to apply the find_window_centroids in both directions (from bottom to top as well as from top to bottom) and choose the better left and right centroids for each level repectively.
np.flipud() is used twice to upside down the binary warped image as an input, as well as the find_window_centroids() output to make the orders of the level in the same direction
In [45]:
def better_window_centroids(binary_warped, window_width, window_height, margin):
"""
The function find window centroids in bidirection (bottom to top & top to bottom)
and then selects better left and right centroids for each level.
"""
# Find window centroids in both directions
window_centroids_up = find_window_centroids(binary_warped, window_width, window_height, margin)
window_centroids_down = np.flipud(find_window_centroids(np.flipud(binary_warped), window_width, window_height, margin))
window_centroids = []
index = []
for level in range(1,int(binary_warped.shape[0]/window_height)):
# Fitting Direction: Bottom -> Top
# Identify window areas
l_mask_up = window_mask(window_width,window_height,binary_warped,window_centroids_up[level][0],level)
r_mask_up = window_mask(window_width,window_height,binary_warped,window_centroids_up[level][1],level)
l_mask_up = np.array(l_mask_up)
r_mask_up = np.array(r_mask_up)
# Identify the nonzero pixels in x and y within the window
good_left_up = (binary_warped*l_mask_up).nonzero()
good_right_up = (binary_warped*r_mask_up).nonzero()
# Fitting Direction: Top -> Bottom
# Identify window areas
l_mask_down = window_mask(window_width,window_height,binary_warped,window_centroids_down[level][0],level)
r_mask_down = window_mask(window_width,window_height,binary_warped,window_centroids_down[level][1],level)
l_mask_down = np.array(l_mask_down)
r_mask_down = np.array(r_mask_down)
# Identify the nonzero pixels in x and y within the window
good_left_down = (binary_warped*l_mask_down).nonzero()
good_right_down = (binary_warped*r_mask_down).nonzero()
# Which is better
# for left
l_center = window_centroids_up[level][0]
if len(good_left_up[0]) < len(good_left_down[0]):
l_center = window_centroids_down[level][0]
# for right
r_center = window_centroids_up[level][1]
if len(good_right_up[0]) < len(good_right_down[0]):
r_center = window_centroids_down[level][1]
window_centroids.append((l_center, r_center))
return window_centroids
In [46]:
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
_, warped_i, _, _ = warp_camera(images[i])
window_centroids_i = better_window_centroids(warped_i, window_width, window_height, margin)
mark_centroids_i = mark_centroids(warped_i, window_centroids_i, window_width, window_height)
r = i//4 # row of image
c = i - r*4 # colomn of image
arr[r,c].imshow(mark_centroids_i, cmap='gray')
arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/better_fitting_result.jpg')
5.2 Find and Plot the Track Line¶
Using the window centroids found by bidirectional window fitting method to plot the track line.
In [47]:
def find_ploty(binary_warped, window_centroids, window_height, window_width):
"""
The function extract left and right line pixel positions
to fit a second order polynomial to both, and generate x and y for plotting.
"""
# Identify the x and y positions of all nonzero pixels in the image
nonzero = binary_warped.nonzero()
nonzeroy = np.array(nonzero[0])
nonzerox = np.array(nonzero[1])
# Set minimum number of pixels found to recenter window
minpix = 100
# Create empty lists to receive valid left and right lane point coordinates
leftx = []
lefty = []
rightx = []
righty = []
# Go through each level to find
for level in range(0, len(window_centroids)):
# Identify window areas
l_mask = window_mask(window_width,window_height,binary_warped,window_centroids[level][0],level)
r_mask = window_mask(window_width,window_height,binary_warped,window_centroids[level][1],level)
l_mask = np.array(l_mask)
r_mask = np.array(r_mask)
# Identify the nonzero pixels in x and y within the window
good_left = (binary_warped*l_mask).nonzero()
good_right = (binary_warped*r_mask).nonzero()
# Append these coordinates to the lists
lefty.append(good_left[0])
leftx.append(good_left[1])
righty.append(good_right[0])
rightx.append(good_right[1])
# Concatenate the arrays of nonzero pixel coordinates
lefty = np.concatenate(lefty)
leftx = np.concatenate(leftx)
righty = np.concatenate(righty)
rightx = np.concatenate(rightx)
# Fit a second order polynomial to each
left_fit = np.polyfit(lefty, leftx, 2)
right_fit = np.polyfit(righty, rightx, 2)
# Generate x and y values for plotting
ploty = np.linspace(0, binary_warped.shape[0]-1, binary_warped.shape[0])
left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
# Pack all x and y coordinates to cut down the numbers of output
lane = np.array([lefty, leftx, righty, rightx])
fitx = np.array([left_fitx, right_fitx])
return ploty, lane, fitx
In [48]:
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
_, warped_i, _, _ = warp_camera(images[i])
window_centroids_i = better_window_centroids(warped_i, window_width, window_height, margin)
ploty_i, lane_i, fitx_i = find_ploty(warped_i, window_centroids_i, window_height, window_width)
r = i//4 # row of image
c = i - r*4 # colomn of image
# Create an output image to draw on and visualize the result
out_img_i = np.dstack((warped_i, warped_i, warped_i))*255
out_img_i[lane_i[0], lane_i[1]] = [255, 0, 0]
out_img_i[lane_i[2], lane_i[3]] = [0, 0, 255]
# Draw Lane Lines
arr[r,c].imshow(out_img_i, cmap='gray', extent=(0,1280,720,0))
arr[r,c].plot(fitx_i[0], ploty_i, color='yellow', linewidth=4)
arr[r,c].plot(fitx_i[1], ploty_i, color='yellow', linewidth=4)
arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/find_ploty.jpg')
6. Determine the Line Shape and Position¶
Determine the curvature of the lane and vehicle position with respect to center.
6.1 Measuring Curvature¶
In [49]:
def calculate_curve(A, B, y):
"""
The function calculate the bottom radius of curvature of a second order
polynomial curve f(y) = A*y**2 + B*y + C.
"""
curve_rad = ((1 + (2*A*y + B)**2)**1.5) / np.absolute(2*A)
return curve_rad
In [50]:
def measure_curve(ploty, lane):
"""
The function calculates the radius of curcature after correcting
for scale in x and y.
"""
# Define y-value where we want radius of curvature
# Here I'll choose the maximum y-value, corresponding to the bottom of the image
y_eval = np.max(ploty)
# Define conversions in x and y from pixels space to meters.
ym_per_pix = 30/720 # meters per pixel in y dimension
xm_per_pix = (3.7/700)*(3/4) # meters per pixel in x dimension
# Fit a second order polynomial to each lane
A_left, B_left, C_left = np.polyfit(lane[0]*ym_per_pix, lane[1]*xm_per_pix, 2)
A_right, B_right, C_right = np.polyfit(lane[2]*ym_per_pix, lane[3]*xm_per_pix, 2)
# Calculate the radius of each lane
left_curverad = calculate_curve(A_left, B_left, y_eval*ym_per_pix)
right_curverad = calculate_curve(A_right, B_right, y_eval*ym_per_pix)
return left_curverad, right_curverad
In [51]:
# Test function
for i in range(len(images)):
undistorted_i, warped_i, M_i, Minv_i = warp_camera(images[i])
window_centroids_i = find_window_centroids(warped_i, window_width, window_height, margin)
ploty_i, lane_i, fitx_i = find_ploty(warped_i, window_centroids_i, window_height, window_width)
left_curverad_i, right_curverad_i = measure_curve(ploty_i, lane_i)
print('['+ names[i] +']')
print('R_left = {:0.2f}(m), R_right = {:0.2f}(m)'.format(left_curverad_i, right_curverad_i))
# Take the average of the left and right curve rad as the lane curveture
curvature_i = (left_curverad_i + right_curverad_i)/2
print('Radius of Curvature = {:0.2f}(m)'.format(curvature_i))
print()
6.2 Determine Vehicle Position¶
In [52]:
def car_position(fitx, x_median):
xm_per_pix = (3.7/700)*(3/4) # meters per pixel in x dimension
car_center = x_median * xm_per_pix # take the image center in x direction as the center of the vehicle
# Take the bottom x positions of the left and right lanes
x_left = fitx[0][-1] * xm_per_pix
x_right = fitx[1][-1] * xm_per_pix
# Take the average of the x_left and x_right as the center of the lane
lane_center = (x_left + x_right)/2
if car_center+0.005 < lane_center:
T_position = 'Vehicle is {:0.2f}m left of center'.format(lane_center-car_center)
return T_position
elif car_center-0.005 > lane_center:
T_position = 'Vehicle is {:0.2f}m right of center'.format(car_center-lane_center)
return T_position
else:
return 'Vehicl is on the center'
In [53]:
x_median = img_lane.shape[1]/2
x_median
Out[53]:
In [54]:
for i in range(len(images)):
undistorted_i, warped_i, M_i, Minv_i = warp_camera(images[i])
window_centroids_i = find_window_centroids(warped_i, window_width, window_height, margin)
ploty_i, lane_i, fitx_i = find_ploty(warped_i, window_centroids_i, window_height, window_width)
T_position_i = car_position(fitx_i, x_median)
print('['+ names[i] +']')
print(T_position_i)
6.3 Drawing Lane¶
In [55]:
def drawing(undistorted_image, binary_warped, ploty, fitx, Minv):
"""
The funtion draw the area between detected lane lines on the undistorted original image.
"""
# Create an image to draw the lines on
warp_zero = np.zeros_like(binary_warped).astype(np.uint8)
color_warp = np.dstack((warp_zero, warp_zero, warp_zero))
# Recast the x and y points into usable format for cv2.fillPoly()
pts_left = np.array([np.transpose(np.vstack([fitx[0], ploty]))])
pts_right = np.array([np.flipud(np.transpose(np.vstack([fitx[1], ploty])))])
pts = np.hstack((pts_left, pts_right))
# Draw the lane onto the warped blank image
cv2.fillPoly(color_warp, np.int_([pts]), (0,255,0))
# Warp the blank back to original image space using inversed perspective matrix (Minv)
newwarp = cv2.warpPerspective(color_warp, Minv, (undistorted_image.shape[1], undistorted_image.shape[0]))
# Combine the result with the original image
result = cv2.addWeighted(undistorted_image, 1, newwarp, 0.3, 0)
return result
In [56]:
# Test function
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
undistorted_i, warped_i, M_i, Minv_i = warp_camera(images[i])
window_centroids_i = find_window_centroids(warped_i, window_width, window_height, margin)
ploty_i, lane_i, fitx_i = find_ploty(warped_i, window_centroids_i, window_height, window_width)
result_i = drawing(undistorted_i, warped_i, ploty_i, fitx_i, Minv_i)
r = i//4 # row of image
c = i - r*4 # colomn of image
arr[r,c].imshow(result_i, cmap='gray')
arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/draw_lane_area.jpg')
In [57]:
def drawing_lane(undistorted_image, binary_warped, lane, Minv):
"""
The funtion draw the detected left and right lane lines on the undistorted original image
in red and blue respectively.
"""
# Create an image to draw the lines on
warp_zero = np.zeros_like(binary_warped).astype(np.uint8)
color_warp = np.dstack((warp_zero, warp_zero, warp_zero))
# draw red left and blue right lane lines
color_warp[lane[0], lane[1]] = [0, 255, 255]
color_warp[lane[2], lane[3]] = [255, 255, 0]
# Warp the blank back to original image space using inversed perspective matrix (Minv)
newwarp = cv2.warpPerspective(color_warp, Minv, (undistorted_image.shape[1], undistorted_image.shape[0]))
# Combine the result with the original image
result = cv2.addWeighted(undistorted_image, 1, newwarp, -0.7, 0)
return result
In [58]:
# Test function
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
undistorted_i, warped_i, M_i, Minv_i = warp_camera(images[i])
window_centroids_i = find_window_centroids(warped_i, window_width, window_height, margin)
ploty_i, lane_i, fitx_i = find_ploty(warped_i, window_centroids_i, window_height, window_width)
result_i = drawing_lane(undistorted_i, warped_i, lane_i, Minv_i)
r = i//4 # row of image
c = i - r*4 # colomn of image
arr[r,c].imshow(result_i, cmap='gray')
arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/draw_lane_line.jpg')
7. Output¶
Warp the detected lane boundaries back onto the original image.
Output visual display of the lane boundaries and numerical estimation of lane curvature and vehicle position.
7.1 Define Function to Process Image¶
In [59]:
# Import everything needed to edit/save/watch video clips
from moviepy.editor import VideoFileClip
from IPython.display import HTML
In [60]:
def process_image(img_camera):
'''
The function:
1. undistorts the original camera image,
2. converts the undistorted image into a binary form via
applying color & gradient thresholds,
3. warps the binary image into birds-eye perspective,
4. detects lane curvature and position via the warped binary image,
5. displays the curvature and position on the undistorted image,
6. draws/marks the detected lane on the undistorted image.
'''
# Undistort the original image to draw lane lines on it later
# Apply color&gradient thresholds and birds-eye perspective on the undistorted image
undistorted_image, binary_warped, M, Minv = warp_camera(img_camera)
# Detect the left and right lane lines
window_centroids = find_window_centroids(binary_warped, window_width=50, window_height=80, margin=40)
ploty, lane, fitx = find_ploty(binary_warped, window_centroids, window_height=80, window_width=50)
# Detect lane curvature and position
left_curverad, right_curverad = measure_curve(ploty, lane)
curvature = (left_curverad + right_curverad)/2
# Display lane curvature and position on the undistorted image
font = cv2.FONT_HERSHEY_SIMPLEX
T_rad = 'Radius of Curvature = {:0.2f}(m)'.format(curvature)
T_position = car_position(fitx, 640)
cv2.putText(undistorted_image, T_rad,(60,100), font, 1.5,(255,255,255), 3, cv2.LINE_AA)
cv2.putText(undistorted_image, T_position,(60,150), font, 1.5,(255,255,255), 3, cv2.LINE_AA)
# Drawing
result = drawing_lane(undistorted_image, binary_warped, lane, Minv)
result = drawing(result, binary_warped, ploty, fitx, Minv)
return result
In [61]:
# Test function
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
result_i = process_image(images[i])
r = i//4 # row of image
c = i - r*4 # colomn of image
arr[r,c].imshow(result_i, cmap='gray')
arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/process_image.jpg')
7.2 Project Video¶
In [62]:
white_output1 = './test_videos_output/project_video.mp4'
# You may also uncomment the following line for a subclip of the first 5 seconds
#clip1 = VideoFileClip("./project_video.mp4").subclip(0,5)
clip1 = VideoFileClip("./project_video.mp4")
white_clip1 = clip1.fl_image(process_image)
%time white_clip1.write_videofile(white_output1, audio=False)
In [63]:
HTML("""
<video width="1280" height="720" controls>
<source src="{0}">
</video>
""".format(white_output1))
Out[63]:
7.3 Challenge Video¶
In [64]:
white_output2 = './test_videos_output/challenge_video.mp4'
# You may also uncomment the following line for a subclip of the first 5 seconds
clip2 = VideoFileClip("./challenge_video.mp4").subclip(0,5)
#clip2 = VideoFileClip("./challenge_video.mp4")
white_clip2 = clip2.fl_image(process_image)
%time white_clip2.write_videofile(white_output2, audio=False)
In [65]:
HTML("""
<video width="1280" height="720" controls>
<source src="{0}">
</video>
""".format(white_output2))
Out[65]:
7.3 Harder Challenge Video¶
In [66]:
white_output3 = './test_videos_output/harder_challenge_video.mp4'
# You may also uncomment the following line for a subclip of the first 5 seconds
clip3 = VideoFileClip("./harder_challenge_video.mp4").subclip(0,5)
#clip3 = VideoFileClip("./harder_challenge_video.mp4")
white_clip3 = clip3.fl_image(process_image)
%time white_clip3.write_videofile(white_output3, audio=False)
In [67]:
HTML("""
<video width="1280" height="720" controls>
<source src="{0}">
</video>
""".format(white_output3))
Out[67]: